#ifndef __PAGELIBPREPROCESSOR_H__
#define __PAGELIBPREPROCESSOR_H__

#include "SplitTool.h"
#include "SplitToolCppJieba.h"
#include "WebPage.h"
#include "Configuration.h"
#include "Configuration.h"
#include "simhash/Simhasher.hpp"

#include <fstream>

using namespace simhash;

using std::ofstream;
using std::ifstream;

class PageLibPreProcessor
{
public :
    PageLibPreProcessor();
    ~PageLibPreProcessor();
    void cutRedudantPage();
    void bulidInvertIndexMap();
    void storeOnDisk();
private :
    vector<WebPage> _pageList;
    vector<uint64_t> _fingerprint;
    vector<pair<int, int>> _offsetLib;
    unordered_map<string, set<pair<int, double>>> _invertIndexLib;
    SplitTool _wordCutter;
};

#endif
